package com.yth.utils;

import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter;
import io.github.bonigarcia.wdm.WebDriverManager;
import lombok.extern.slf4j.Slf4j;
import org.openqa.selenium.By;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * Web内容获取并转换为Markdown的工具类。
 * 支持静态HTML页面、API接口、需要JS渲染的现代化网页。
 *
 * @author yutianhong
 * @version 1.0
 * @since 2025/7/10 17:32
 */
@Slf4j
public class UrlToMarkdownConverter implements AutoCloseable {

    private static final int DEFAULT_TIMEOUT_SECONDS = 10;
    private static final int DEFAULT_JS_DELAY_MS = 2000;
    private static final int MAX_WEB_DRIVERS = 3;
    private static final String DEFAULT_USER_AGENT =
            "Mozilla/5.0 (Macintosh; Intel Mac OS X10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";

    // JS脚本：移除全局所有不需要的节点
    private static final String CLEAN_SCRIPT =
            "var sel=['script','style','nav','footer','header','aside','.nav','.footer','.header','.sidebar','.navigation'," +
                    "'[class*=advertisement]','[id*=advertisement]','[class*=popup]','[id*=popup]','[class*=modal]','[id*=modal]'," +
                    "'*[style*=\"display:none\"]','*[style*=\"visibility:hidden\"]','img','svg'];" +
                    "sel.forEach(function(s){document.querySelectorAll(s).forEach(function(n){n.parentNode.removeChild(n);});});";

    // 移除注释节点
    private static final String REMOVE_COMMENTS_SCRIPT =
            "try{var comments=[];" +
                    "var treeWalker=document.createTreeWalker(document.body,NodeFilter.SHOW_COMMENT,null,false);" +
                    "while(treeWalker.nextNode()){comments.push(treeWalker.currentNode);}" +
                    "comments.forEach(function(n){if(n&&n.parentNode){n.parentNode.removeChild(n);}});" +
                    "}catch(e){}";

    // 选择主内容并返回innerHTML，如果没有主内容，返回body内容
    private static final String EXTRACT_MAIN_CONTENT_SCRIPT =
            "var mains=['article','main','[role=main]','.content','#content','.post-content','.entry-content','.article-content','.markdown-body'];" +
                    "for(var i=0;i<mains.length;i++){" +
                    "try{var el=document.querySelector(mains[i]);" +
                    "if(el&&el.innerText&&el.innerText.trim().length>200){return el.innerHTML;}" +
                    "}catch(e){}" +
                    "}" +
                    "return document.body?document.body.innerHTML:'';";

    private final int timeoutSeconds;
    private final int jsDelayMs;
    private final WebDriverPool webDriverPool;
    private final FlexmarkHtmlConverter htmlConverter;

    static {
        // 初始化ChromeDriver
        // 这操作有一定的重量级，但是如果不提前初始化，那么启动时延会特别大。
        WebDriverManager.chromedriver().setup();
    }

    /**
     * 使用默认配置创建转换器
     */
    public UrlToMarkdownConverter() {
        this(DEFAULT_TIMEOUT_SECONDS, DEFAULT_JS_DELAY_MS, MAX_WEB_DRIVERS);
    }

    /**
     * 使用自定义配置创建转换器
     *
     * @param timeoutSeconds 页面加载超时时间（秒），最小值为1
     * @param jsDelayMs      JavaScript执行等待时间（毫秒），最小值为0
     * @param maxWebDrivers  maxWebDrivers 最大WebDriver数量
     */
    public UrlToMarkdownConverter(int timeoutSeconds, int jsDelayMs, int maxWebDrivers) {
        this.timeoutSeconds = Math.max(1, timeoutSeconds);
        this.jsDelayMs = Math.max(0, jsDelayMs);
        this.webDriverPool = new WebDriverPool(maxWebDrivers);
        this.htmlConverter = FlexmarkHtmlConverter.builder().build();
    }

    /**
     * 将URL内容转换为Markdown格式
     *
     * @param url 要转换的URL
     * @return Markdown格式的内容
     */
    public String convertToMarkdown(String url) {
        String normalizedUrl = normalizeUrl(url);
        try (WebDriverResource webDriverResource = webDriverPool.borrowDriver()) {
            WebDriver webDriver = webDriverResource.getDriver();
            webDriver.get(normalizedUrl);
            waitForContentLoad(webDriver);
            String title = webDriver.getTitle();
            String cleanHtml = cleanAndExtractContent(webDriver);
            if (title.isEmpty()) {
                return htmlConverter.convert(cleanHtml);
            } else {
                return "# " + title + "\n\n" + htmlConverter.convert(cleanHtml);
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new ContentFetchException("Content fetching was interrupted", e);
        } catch (Throwable e) {
            throw new ContentFetchException("Failed to convert URL to markdown: " + normalizedUrl + " cause by " + e.getMessage(), e);
        }
    }

    /**
     * 关闭资源
     */
    @Override
    public void close() {
        if (webDriverPool != null) {
            webDriverPool.close();
        }
    }

    private static String normalizeUrl(String url) {
        if (url == null || url.trim().isEmpty()) {
            throw new IllegalArgumentException("URL cannot be null or empty");
        }

        String normalizedUrl = url.trim();

        // 如果没有协议，添加http://
        if (!normalizedUrl.startsWith("http://") && !normalizedUrl.startsWith("https://")) {
            normalizedUrl = "http://" + normalizedUrl;
        }

        // 验证URL格式
        try {
            new URL(normalizedUrl);
            return normalizedUrl;
        } catch (Throwable e) {
            throw new IllegalArgumentException("Invalid URL format: " + url, e);
        }
    }

    private void waitForContentLoad(WebDriver driver) throws InterruptedException {
        WebDriverWait wait = new WebDriverWait(driver, timeoutSeconds);

        // 等待主要内容加载
        try {
            wait.until(ExpectedConditions.presenceOfElementLocated(By.tagName("body")));
        } catch (Exception ignored) {
        }
        // 额外等待确保JS执行完成
        if (jsDelayMs > 0) {
            Thread.sleep(jsDelayMs);
        }
    }

    /**
     * 清理文档并提取主要内容
     */
    private static String cleanAndExtractContent(WebDriver webDriver) {
        ((JavascriptExecutor) webDriver).executeScript(CLEAN_SCRIPT);
        ((JavascriptExecutor) webDriver).executeScript(REMOVE_COMMENTS_SCRIPT);
        return (String) ((JavascriptExecutor) webDriver).executeScript(EXTRACT_MAIN_CONTENT_SCRIPT);
    }

    /**
     * WebDriver对象池
     */
    private static class WebDriverPool {
        private final BlockingQueue<WebDriver> availableDrivers;
        private final Set<WebDriver> allDrivers;
        private final int maxSize;
        private final AtomicBoolean closed = new AtomicBoolean(false);

        public WebDriverPool(int maxSize) {
            this.availableDrivers = new ArrayBlockingQueue<>(maxSize);
            this.allDrivers = ConcurrentHashMap.newKeySet((int) (maxSize / 0.75) + 1);
            this.maxSize = maxSize;
        }

        public WebDriverResource borrowDriver() throws InterruptedException {
            if (closed.get()) {
                throw new IllegalStateException("WebDriverPool is closed.");
            }
            WebDriver driver = availableDrivers.poll(500, TimeUnit.MILLISECONDS);
            if (driver != null) {
                return new WebDriverResource(this, driver);
            }
            if (allDrivers.size() < maxSize) {
                synchronized (this) {
                    if (allDrivers.size() < maxSize) {
                        WebDriver newDriver = createWebDriver();
                        allDrivers.add(newDriver);
                        return new WebDriverResource(this, newDriver);
                    }
                }
            }

            // 如果池满了，创建临时driver
            WebDriver newDriver = createWebDriver();
            return new WebDriverResource(this, newDriver);
        }

        public void returnDriver(WebDriver driver) {
            if (driver != null && !closed.get() && allDrivers.contains(driver)) {
                cleanDriverState(driver);
                boolean success = availableDrivers.offer(driver);
                if (success) {
                    return;
                }
            }
            if (driver != null) {
                safeQuitDriver(driver);
            }
        }

        public void close() {
            if (closed.compareAndSet(false, true)) {
                for (WebDriver driver : allDrivers) {
                    safeQuitDriver(driver);
                }
                allDrivers.clear();
                availableDrivers.clear();
            }
        }

        private void safeQuitDriver(WebDriver driver) {
            try {
                driver.quit();
            } catch (Exception e) {
                log.warn("关闭WebDriver时发生异常", e);
            }
        }

        private void cleanDriverState(WebDriver driver) {
            try {
                driver.manage().deleteAllCookies();
                if (driver instanceof JavascriptExecutor) {
                    JavascriptExecutor js = (JavascriptExecutor) driver;
                    js.executeScript("try { localStorage.clear(); } catch(e) {}");
                    js.executeScript("try { sessionStorage.clear(); } catch(e) {}");
                }
            } catch (Exception e) {
                log.warn("清理WebDriver状态失败", e);
            }
        }

        private WebDriver createWebDriver() {
            ChromeOptions options = new ChromeOptions()
                    .addArguments("--headless")               // 无头模式
                    .addArguments("--no-sandbox")             // 绕过OS安全模型
                    .addArguments("--disable-dev-shm-usage")  // 解决共享内存问题
                    .addArguments("--disable-gpu")            // 禁用GPU硬件加速
                    .addArguments("--window-size=1920,1080")  // 设置窗口大小
                    .addArguments("--disable-web-security")   // 禁用Web安全检查
                    .addArguments("--disable-images")         // 禁用图片加载提升性能
                    .addArguments("--user-agent=" + DEFAULT_USER_AGENT);
            return new ChromeDriver(options);
        }
    }

    private static class WebDriverResource implements AutoCloseable {
        private final WebDriverPool webDriverPool;
        private final WebDriver driver;

        WebDriverResource(WebDriverPool webDriverPool, WebDriver driver) {
            this.webDriverPool = Objects.requireNonNull(webDriverPool);
            this.driver = Objects.requireNonNull(driver);
        }

        public WebDriver getDriver() {
            return driver;
        }

        @Override
        public void close() {
            webDriverPool.returnDriver(driver);
        }
    }

    /**
     * 内容获取异常（运行时异常）
     */
    public static class ContentFetchException extends RuntimeException {
        public ContentFetchException(String message, Throwable cause) {
            super(message, cause);
        }
    }

    public static void main(String[] args) {
        try (UrlToMarkdownConverter urlToMarkdownConverter = new UrlToMarkdownConverter()) {
            List<String> urls = Arrays.asList(
                    "https://httpbin.org/html",
                    "https://mocks.alibaba-inc.com/mock/ythtest//appIteration/page",
                    "https://www.baidu.com",
                    "https://keepachangelog.com/zh-CN/1.1.0/",
                    "https://docs.langchain4j.info/intro",
                    "https://www.baeldung.com/jackson"
            );
            for (String url : urls) {
                System.out.println("========================================================================================");
                long startTime = System.currentTimeMillis();
                try {
                    String markdown = urlToMarkdownConverter.convertToMarkdown(url);
                    System.out.println("耗时: " + (System.currentTimeMillis() - startTime) + "ms");
                    System.out.println(url + "转换结果:");
                    System.out.println(markdown);
                } catch (Throwable e) {
                    System.err.println("错误信息: " + e.getMessage());
                    e.printStackTrace();
                }
                System.out.println("========================================================================================");
            }
            System.out.println(urlToMarkdownConverter);
        }
    }
}
